from pathlib import Path
import random
import time
import itertools
import re
import csv
import pandas
from openpyxl import load_workbook
import pyexcel

from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup

from browser import Browser
from append_df import append_df_to_excel



class Parser(Browser):

    """ Этот класс представляет парсер """

    def __init__(self, email, password, display_browser):

        self.link = 'http://msk.galara.ru/'
        self.email = email
        self.password = password
        self.display_browser = display_browser


    def wait(self, f=2, b=3.5):

        """ Чуть-чуть подождать """

        number = random.uniform(f, b)
        print(f'Ждем {number}\n')
        time.sleep(number)


    def wirte_csv_minimum_price(self, data):

        with open('data.csv', 'a') as file:

            writer = csv.writer(file)
            writer.writerow((
                data[0],
                data[1],
                data[2],
                data[3],
                data[4],
                data[5]
            ))


    def write_csv(self, data):

        """ Записываем данные в csv файл """

        with open('data.csv', 'a') as file:

            writer = csv.writer(file)
            writer.writerow((
                data[0],
                data[1],
                data[2],
                data[3],
                data[4]
            ))


    def convert_to_bs(self, browser):

        """ Получаем объект BeautifulSoup """

        html_source = browser.page_source
        page = BeautifulSoup(html_source, 'lxml')

        return page


    def get_saved_page(self, file_name):

        """ Открываем сохраненную страницу для парсинга и возвращаем ее """

        folder = Path('saved_html/')
        page = folder/file_name

        with open(page, 'r', encoding='utf-8') as file:

            text = file.read()
            page = BeautifulSoup(text, 'lxml')

            return page


    def login(self, browser, link):

        """ Авторизуемся на сайте """

        browser.get(link)

        email = browser.find_element(
            By.XPATH,
            '//div[@class="left-column"]/div[1]//form//input[@id="maemail"]'
        )
        email.send_keys(self.email)

        self.wait(1, 1.5)

        password = browser.find_element(
            By.XPATH,
            '//div[@class="left-column"]/div[1]//form//input[@id="mapass"]'
        )
        password.send_keys(self.password)

        enter = browser.find_element(
            By.XPATH,
            '//div[@class="left-column"]/div[1]//form//input[@value="Войти"]'
        ).click()

        self.wait(2, 3)


    def start_parser(self):

        """ Начать парсинг """

        def get_all_brands_letter(brand, browser):

            """ Получаем все бренды буквы """

            def add_products_to_brand(catalog, products):

                """ Добавляем все продукты в бренд """

                catalog.append(products)
                #for brand, products in itertools.zip_longest(catalog, products):

                    #brand.append(products)


            # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

            result = []
            #for product in catalog:

            browser.get(brand[1])

            titles = browser.find_elements(
                By.XPATH,
                '//div[@class="main-column"]//table[@class="brands"]//div/a'
            )
            ready_titles = [item.text for item in titles]

            links = browser.find_elements(
                By.XPATH,
                '//div[@class="main-column"]//table[@class="brands"]//div//a'
            )
            ready_links = [
                item.get_attribute('href')
                for item in links
            ]

            done = []

            for (title, link) in itertools.zip_longest(ready_titles, ready_links):

                data = [title, link]
                done.append(data)

            result.append(done)
            self.wait(1.4, 2)

            result = result[0]
            add_products_to_brand(brand, result)

            return brand


        def get_brand_catalog():

            """ Получаем каталог брендов """

            browser = self.get_browser(self.display_browser)
            browser.get(self.link)
            page = self.convert_to_bs(browser)

            brand_catalog = page.find('div', {'class': 'left-column'}) \
                .find('div', {'class': 'alpha-block'}) \
                .find_all('span')

            result = [
                [element.find('a').text, 'http://msk.galara.ru' + element.find('a')['href']]
                for element in brand_catalog
            ]

            browser.quit()
            return result


        def get_second_product_name(brands, browser):

            """ Работаем со всеми продуктами """

            def go_to_brand(products):

                """ Работаем с брендом """

                def get_all_products(link):

                    """ Получаем все продукты """

                    browser.get(link)

                    a = browser.find_elements(
                        By.XPATH,
                        '//div[@id="list_brand_lines"]/div/div[@class="name"]//a'
                    )

                    titles = [item.text for item in a]
                    links = [item.get_attribute('href') for item in a]

                    return (titles, links)


                def join_lists(lists):

                    """ Объединяем списки """

                    result = []
                    titles, links = lists

                    for title, link in itertools.zip_longest(titles, links):

                        data = [title, link]
                        result.append(data)

                    return result


                def get_all_information(products):

                    """ Получаем всю информацию с продукта """

                    def get_title(product):

                        """ Получаем название продукта """

                        title = browser.find_element(
                            By.XPATH,
                            '//div[@class="main-column"]//h1'
                        ).text

                        return title


                    def get_first_group_parameters(product):

                        """ Получаем первую группу параметров """

                        result = []

                        parameters_one = browser.find_elements(
                            By.XPATH,
                            '//div[@class="product-char i-block"]/div[@class="name"]'
                        )
                        done_parameters_one = [item.text for item in parameters_one]

                        parameters_two = browser.find_elements(
                            By.XPATH,
                            '//div[@class="product-char i-block"]/div[@class="items"]'
                        )
                        done_parameters_two = [item.text for item in parameters_two]


                        for (p_one, p_two) in itertools.zip_longest(done_parameters_one, done_parameters_two):

                            data = [p_one, p_two]
                            result.append(data)

                        return result


                    def get_second_group_parameters(product):

                        """ Получаем вторую группу параметров """

                        def get_additionally():

                            """ Получаем дополнительную информацию о продукте """

                            done_additionally = []

                            page = self.convert_to_bs(browser)
                            additionallys = page.find('table', {'class': 'list-price'}) \
                                .find_all('tr')[1:]

                            for item in additionallys:

                                item = item.contents[0]

                                for td in item:

                                    try:

                                        result = td.find('div').text

                                    except:

                                        result = ''

                                    done_additionally.append(result)

                            return done_additionally


                        # ВЫШЕ ОБЪЯВЛЕНИЯ ФУНКЦИЙ

                        result = []
                        done_volumes = []

                        volumes = browser.find_elements(
                            By.XPATH,
                            '//table[@class="list-price"]//td[2]'
                        )

                        for volume in volumes:

                            try:

                                done_volumes.append(volume.text)

                            except:

                                done_volumes.append('Объем не указан')


                        done_additionally = get_additionally()

                        price = browser.find_elements(
                            By.XPATH,
                            '//table[@class="list-price"]//td[3]'
                        )
                        done_price = [item.text for item in price]

                        title = browser.find_elements(
                            By.XPATH,
                            '//table[@class="list-price"]//td[1]'
                        )
                        done_title = [item.text for item in title]


                        for (volume, price, title, additionally) in itertools.zip_longest(done_volumes, done_price, done_title, done_additionally):

                            data = [volume, price, title, additionally]
                            result.append(data)

                        return result


                    def get_all_images(product):

                        """ Получаем все изображения """

                        try:

                            main_image = browser.find_element(
                                By.XPATH,
                                '//div[@class="product-img i-block"]//div[@class="img data-big-photo"]//img'
                            ).get_attribute('src')
                            done_main_image = main_image

                            other_images = browser.find_elements(
                                By.XPATH,
                                '//div[@class="product-img i-block"]//table[@class="data-list"]//img'
                            )
                            ready_other_image = [item.get_attribute('src') for item in other_images]

                            ready_other_image.insert(0, done_main_image)

                            return ready_other_image

                        except Exception as e:

                            print(e)
                            return ['']


                    # ВЫШЕ ОБЪЯВЛЕНИЕ ФУНКЦИЙ

                    done = []

                    for product in products:

                        browser.get(product[1])

                        first_group_parameters = get_first_group_parameters(product)
                        second_group_parameters = get_second_group_parameters(product)
                        all_images = get_all_images(product)
                        title = get_title(product)
                        data = (title, first_group_parameters, second_group_parameters, all_images)
                        done.append(data)

                        self.wait(1.5, 2.2)

                    return done


                done = []

                # ВЫШЕ ОБЪЯВЛЕНИЕ ФУНКЦИЙ

                for product in products:

                    title = product[0]
                    link = product[1]

                    print(f'Все продукты бренда {title}')

                    all_products = get_all_products(link)
                    combined_lists = join_lists(all_products)
                    all_information = get_all_information(combined_lists)

                    all_information.insert(0, title)
                    done.append(all_information)

                return done


            # ВЫШЕ ОБЪЯВЛЕНИЕ ФУНКЦИЙ

            done = []

            #for letter in products:

            main = [brands[0], brands[1]]
            products = brands[2]
            d = go_to_brand(products)
            #d.insert(0, products)
            done.append(d)
            done.insert(0, main)

            return done


        def sort_data(brand):

            def add_lowest_price(description):

                """ Выявляем меньшую цену """

                if not ';' in description[1]:

                    price = description[1].split(' ')[-1]
                    price = int(price)
                    return price

                prices = description[1].split(';')

                temp_prices = []

                for price in prices:

                    price = int(price.split(' ')[-1])
                    temp_prices.append(price)

                result = min(temp_prices)

                return result


            def get_description_one(product):

                """ Получаем описание продукта """

                result = []

                for parameters in product:

                    done = ''.join(parameters)
                    result.append(done)

                result = ', '.join(result)

                return result


            def get_description_two(product):

                """ Получаем параметры продукта """

                done = []
                result = []

                for item in product:

                    title = item[2].strip()
                    result.append(title)

                result = list(set(result))
                result = [[item] for item in result]

                for i in result:

                    for item in product:

                        if 'нет в наличии' in item[1]: continue

                        title = item[2].strip()
                        price = re.findall(r'\d', item[1])
                        price = ''.join(price)

                        if title == i[0]: i.append([item[3], item[0], price])


                for item in result:

                    if len(item) == 1: continue
                    done.append(item)


#                for item in done:
#
#                    title = item[0]
#                    desc = [i for i in item[1] if i]
#                    desc = ' - '.join(desc)


                return done


            def get_full_name_and_description(name, description):

                """ Получаем полное название и описание """

                done = []
                result = []

                for item in description:

                    title = item[0]
                    full_title = f'{name} {title}'
                    result.append([full_title, item[1:]])

                for item in result:

                    for j in item[1]:

                        done.append([item[0], ' - '.join(j)])

                return done


            def get_description(description):

                """ Получаем готовое описание """

                products = []

                for item in description:

                    products.append(item[0])

                products = list(set(products))
                products = [[item] for item in products]

                for product in products:

                    for item in description:

                        title = item[0]
                        #if item[1][1] == '-': desc = item[1][3:]
                        if product[0] == title: product.append(item[1:])

                return products


            # ВЫШЕ ОБЪЯВЛЕНИЕ ФУНКЦИЙ

            for products in brand[1]:

                letter = products[0][0]
                product_name_one = products[0]
                product_name_two = products[1][0]
                description_one = products[1][1]
                description_two = products[1][2]
                print('brand[1]')
                print(brand[1])
                print('brand[1]')

                for product in products:

                    if not type(product) == tuple: continue
                    if 'нет в наличии' in product[2][0][1]: continue

                    title = product[0]
                    done_description_one = get_description_one(product[1])
                    done_description_two = get_description_two(product[2])

                    #title = f'{product_name_one} {product_name_two}'
                    description = get_full_name_and_description(title, done_description_two)
                    products = get_description(description)

                    bla = []

#                    print('description')
#                    print(description)
#                    print('description')
#                    print('products')
#                    print(products)
#                    print('products')
                    for p in products:

                        title = p[0]
                        description = [item[0] for item in p[1:]]
                        done = []
                        for i in description:
                            if i[1] == '-': done.append(i[3:])
                        description = '; '.join(done)
                        bla.append([title, description])


                    for p in bla:

                        title = p[0]
                        description = p[1]
                        images = product[3]
                        images = ', '.join(images)
                        min_number = add_lowest_price(p)
                        #print(f'{title} {description} {images} {min_number}')

                        df = pandas.DataFrame({'Буква': [letter],
                                            'Название': [title],
                                            'Характеристики': [done_description_one],
                                            'Изображения': [images],
                                            'Параметры': [description],
                                            'Минимальная цена': [min_number]})

                        append_df_to_excel('./data.xlsx', df)

                        print('Записали!!!!!!!')


        def convert_data():

            """ Преобразуем excle файл в нормальный вид """

            result = []
            letter_l = []
            title_l = []
            descriptions_l = []
            images_l = []
            parameters_l = []
            prices_l = []

            try:

                data = pyexcel.get_array(file_name='data.xlsx')

            except:

                print('Продуктов для записис нет')
                return

            print(data)

            for product in data:

                if product[0] == '': continue

                letter = product[1]
                title = product[2]
                description = product[3]
                images = product[4]
                parameters = product[5]
                price = product[6]

                letter_l.append(letter)
                title_l.append(title)
                descriptions_l.append(description)
                images_l.append(images)
                parameters_l.append(parameters)
                prices_l.append(price)

            df = pandas.DataFrame({'Буква': letter_l,
                                   'Название': title_l,
                                   'Описание': descriptions_l,
                                   'Изображения': images_l,
                                   'Параметры': parameters_l,
                                   'Цена': prices_l
                                   })
            df.to_excel('./done_data.xlsx')


        # ВЫШЕ ОПРЕДЕЛЕНИЕ ФУНКЦИЙ

        brand_catalog = get_brand_catalog()    # Получаем список брендов

        #brand_catalog.reverse()
        #brand_catalog = brand_catalog[3:]

        for i, brand in enumerate(brand_catalog, 1):

            #print(brand[0])
            browser = self.get_browser(self.display_browser)
            print('Получили браузер')
            self.login(browser, brand[1])
            print('Авторизировались')

            brands_letter = get_all_brands_letter(brand, browser)
            brand_products = get_second_product_name(brands_letter, browser)
            sort_data(brand_products)
            convert_data()

            browser.quit()
            print('Закрыли')
